Lyrics Analysis

Lets us read the file.

library(readr)
lyrics <- read_csv("songdata.csv")
head(lyrics)

Lets us examine the dimension of the lyrics dataframe.

dim(lyrics)

[1] 57650     4

library(dplyr)

Attaching package: 'dplyr'

The following objects are masked from 'package:stats':

    filter, lag

The following objects are masked from 'package:base':

    intersect, setdiff, setequal, union

glimpse(lyrics)

Observations: 57,650
Variables: 4
$ artist <chr> "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "ABBA", "A...
$ song   <chr> "Ahe's My Kind Of Girl", "Andante, Andante", "As Good As New", "Bang", "Bang-A-Boomerang", "Burnin...
$ link   <chr> "/a/abba/ahes+my+kind+of+girl_20598417.html", "/a/abba/andante+andante_20002708.html", "/a/abba/as...
$ text   <chr> "Look at her face, it's a wonderful face  \nAnd it means something special to me  \nLook at the wa...

Analysis of 55000+ lyrics data - Number of artists - Which artist has highest and lowest number of songs - Distribution of songs of all artists in the dataset - Distribution of lyrics length - Which song lyrics has maximum number of words - Which song lyrics has minimum number of words - Distribution of words count in title - Which songs title has maximum number of words - Which songs title has minimum number of words - WordClouds of titles with minimum and maximum lengths - Is there a relation between title length and song length?

• Sentiments of the songs (NRC, Bing)
• Which words are most occuring in the lyrics of the songs
• Is there a correlation between the words in the songs of same artists?
• Wordcloud of most popular words in the songs
• Top words used by an artist in his/her songs
• Are there some common Rythmic words that repeats again and again?

Let’s start with finding out how many artists are listed in the data. Also, how many songs each artist has.

artist<- as.data.frame(table(as.data.frame(lyrics$artist)))
colnames(artist) <- c("artist", "Num_of_songs")
head(artist)

Let’s see the which artist has most and least number of songs in the dataset.

most_songs <- arrange(artist, desc(Num_of_songs))
most_songs

Loading required package: lattice
Loading required package: plyr
--------------------------------------------------------------------------------------------------------------------
You have loaded plyr after dplyr - this is likely to cause problems.
If you need functions from both plyr and dplyr, please load plyr first, then dplyr:
library(plyr); library(dplyr)
--------------------------------------------------------------------------------------------------------------------

Attaching package: 'plyr'

The following objects are masked from 'package:dplyr':

    arrange, count, desc, failwith, id, mutate, rename, summarise, summarize

least_songs <- tail(most_songs, 15)
p2 <- ggplot(data = least_songs, aes(artist, Num_of_songs, fill = Num_of_songs)) +
      geom_bar(stat = "identity") +
      geom_text(aes(label=Num_of_songs), vjust=1.6, color="white", size=3) +
      ggtitle("Artists with least number of songs") +
      tilt_theme
p2

Let’s check the distribution of songs for all artists.

p3 <- ggplot(artist, aes(x=Num_of_songs)) + 
 geom_histogram(aes(y=..density..), colour="black", fill="white")+
 geom_density(alpha=.2, fill="red")
p3

Let’s analyze the number of words in each song and its distribution.

library(stringr)
count_words <- function(vec){
  return (length(unlist((str_extract_all(tolower(vec), '\\w+')))))
}
lyrics$word_count <- sapply(lyrics$text, count_words)
head(lyrics$word_count)

[1] 161 272 322 257 255 115

p4 <- ggplot(lyrics, aes(x=word_count)) + 
 geom_histogram(aes(y=..density..), colour="black", fill="white")+
 geom_density(alpha=.2, fill="red")
p4

Let’s analyze the title of the songs, their wordcount and their distribution

lyrics$title_word_count <- sapply(lyrics$song, count_words)
head(lyrics$title_word_count)

[1] 6 2 4 1 3 3

Let’s check out the songs that are longest and shortest.

longest_song <- arrange(lyrics, desc(word_count))
longest_song <- head(longest_song, 10)
shortest_song <- arrange(lyrics, word_count)
shortest_song <- head(shortest_song, 10)
longest_song

shortest_song

p5 <- ggplot(data = longest_song, aes(song, word_count, fill = title_word_count)) +
      geom_bar(stat = "identity") +
      geom_text(aes(label=title_word_count), vjust=1.6, color="white", size=3) +
      ggtitle("Longest Songs") +
      tilt_theme
p6 <- ggplot(data = shortest_song, aes(song, word_count, fill = title_word_count)) +
      geom_bar(position = "dodge", stat = "identity") +
      geom_text(aes(label = title_word_count), vjust = 1.6, color = "white", size = 3) +
      ggtitle("Shortest Songs") +
      tilt_theme
multiplot(p5, p6, cols=2)

p7 <- ggplot(lyrics, aes(x=title_word_count)) + 
 geom_histogram(aes(y=..density..), colour="black", fill="white", binwidth = 1, bins = 1)+
 geom_density(alpha=.2, fill="red")
p7

WordCloud of popular words from song titles

library(wordcloud)

Loading required package: RColorBrewer

library(SnowballC)
library(RColorBrewer)
library(tm)

Loading required package: NLP

Attaching package: 'NLP'

The following object is masked from 'package:ggplot2':

    annotate

texts <- lyrics$song
#texts <- iconv(texts, to = "utf-8")
corpus <- Corpus(VectorSource(texts))
corpus <- tm_map(corpus, PlainTextDocument)
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords, stopwords('english'))
corpus <- tm_map(corpus, stemDocument)
corpus <- tm_map(corpus, removeWords, c("and", "this", "there")) 
corpus <- Corpus(VectorSource(corpus))
dtm <- TermDocumentMatrix(corpus)
m <- as.matrix(dtm)
v <- sort(rowSums(m),decreasing=TRUE)
d <- data.frame(word = names(v),freq=v)
head(d, 10)

d <- d[-which(d$word %in% c("and","this","that")),]
set.seed(1234)
wordcloud(words = d$word, freq = d$freq, min.freq = 1,
          max.words=200, random.order=FALSE, rot.per=0.35, 
          colors=brewer.pal(8, "Dark2"))

There are many song titles that are of length 1, 2 and 3. But surprisingly, there are titles of length more than 13 too. Let’s check them out.

longest_title <- subset(lyrics, lyrics$title_word_count > 13)
longest_title

shortest_title <- subset(lyrics, lyrics$title_word_count == 1)
shortest_title

There are 8 songs with title length more than 13 and 8342 songs with single word title. Let’s see word cloud of single word titles and longest titles

texts <- longest_title$song
corpus <- Corpus(VectorSource(texts))
corpus <- tm_map(corpus, PlainTextDocument)
corpus <- Corpus(VectorSource(corpus))
dtm <- TermDocumentMatrix(corpus)
m <- as.matrix(dtm)
v <- sort(rowSums(m),decreasing=TRUE)
d <- data.frame(word = names(v),freq=v)
head(d, 10)

set.seed(1234)
wordcloud(words = d$word, freq = d$freq, min.freq = 1,scale=c(2,0.5),
          max.words=100, random.order=FALSE, rot.per=0.35, 
          colors=brewer.pal(8, "Dark2"))

texts <- shortest_title$song
corpus <- Corpus(VectorSource(texts))
corpus <- tm_map(corpus, PlainTextDocument)
corpus <- Corpus(VectorSource(corpus))
dtm <- TermDocumentMatrix(corpus)
m <- as.matrix(dtm)
v <- sort(rowSums(m),decreasing=TRUE)
d <- data.frame(word = names(v),freq=v)
head(d, 10)

set.seed(1234)
wordcloud(words = d$word, freq = d$freq, min.freq = 1,scale=c(2,0.5),
          max.words=100, random.order=FALSE, rot.per=0.35, 
          colors=brewer.pal(8, "Dark2"))

An interesting questin would be is there relation between length of title and songs? Most probably now, but let’s check out.

p8 <- ggplot(lyrics, aes(x=factor(title_word_count), y=word_count, fill = factor(title_word_count))) + 
  geom_boxplot() 
p8 

cor(lyrics$title_word_count, lyrics$word_count)

[1] -0.02509779

As expected, there is no correlation between these two quantitites.

Let us fix the contracted words to their full forms first.

# function to expand contractions in an English-language source
fix.contractions <- function(doc) {
  # "won't" is a special case as it does not expand to "wo not"
  doc <- gsub("won't", "will not", doc)
  doc <- gsub("can't", "can not", doc)
  doc <- gsub("n't", " not", doc)
  doc <- gsub("'ll", " will", doc)
  doc <- gsub("'re", " are", doc)
  doc <- gsub("'ve", " have", doc)
  doc <- gsub("'m", " am", doc)
  doc <- gsub("'d", " would", doc)
  # 's could be 'is' or could be possessive: it has no expansion
  doc <- gsub("'s", "", doc)
  return(doc)
}
# fix (expand) contractions
lyrics$text <- sapply(lyrics$text, fix.contractions)

Remove special characters from lyrics

# function to remove special characters
removeSpecialChars <- function(x) gsub("[^a-zA-Z0-9 ]", " ", x)
# remove special characters
lyrics$text <- sapply(lyrics$text, removeSpecialChars)

Convert all lyrics text to lower case

# convert everything to lower case
lyrics$text <- sapply(lyrics$text, tolower)

Let’s check the structure of one lyrics to see the changes.

str(lyrics[13, ]$text, nchar.max = 300)

 chr "changing  moving in a circle   i can see your face in all of my dreams   smiling  laughing from the shadows   when i hear your voice  i know what it means   i know it does not matter just how hard i try   you are all the reason for my life      disillusion  disillusion all you left "| __truncated__

SENTIMENT ANALYSIS OF LYRICS Let us perform sentiment analysis on the lyrics. There are various types of sentiment lexicons that can be used. Lets us have a look on them.

library(tidytext)

package 'tidytext' was built under R version 3.4.4

library(tidyr)
get_sentiments("afinn")

get_sentiments("bing")

get_sentiments("nrc")

get_sentiments("loughran")

nrc seems to have large number of words and their sentiments compared to other two.

nrc_sentiment <- get_sentiments("nrc")
unique(nrc_sentiment$sentiment)

 [1] "trust"        "fear"         "negative"     "sadness"      "anger"        "surprise"     "positive"    
 [8] "disgust"      "joy"          "anticipation"

Let us find the sentiments of each lyrics based on each NRC sentiments.

lyrics_words <- select(lyrics, c("artist", "text"))
lyrics_words <- lyrics_words %>% unnest_tokens(word, text)
head(lyrics_words)

dim(lyrics_words)

[1] 13186646        2

Let’s see words that depict “joy”

joy <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "joy")) 

Joining, by = "word"

joy <- as.data.frame(sort(table(joy$word)))
columns_sentiment <- c("word", "Freq")
colnames(joy) <- columns_sentiment
tail(joy, 10)

Similarly, let’s see other words that represents other 9 sentiments.

trust <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "trust")) 

Joining, by = "word"

trust <- as.data.frame(sort(table(trust$word)))
colnames(trust) <- columns_sentiment
tail(trust, 10)

fear <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "fear")) 

Joining, by = "word"

fear <- as.data.frame(sort(table(fear$word)))
colnames(fear) <- columns_sentiment
sadness <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "sadness")) 

Joining, by = "word"

sadness <- as.data.frame(sort(table(sadness$word)))
colnames(sadness) <- columns_sentiment
anger <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "anger")) 

Joining, by = "word"

anger <- as.data.frame(sort(table(anger$word)))
colnames(anger) <- columns_sentiment
surprise <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "surprise")) 

Joining, by = "word"

surprise <- as.data.frame(sort(table(surprise$word)))
colnames(surprise) <- columns_sentiment
disgust <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "disgust")) 

Joining, by = "word"

disgust <- as.data.frame(sort(table(disgust$word)))
colnames(disgust) <- columns_sentiment
anticipation <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "anticipation")) 

Joining, by = "word"

anticipation <- as.data.frame(sort(table(anticipation$word)))
colnames(anticipation) <- columns_sentiment

Let’s plot the word occurences for each sentiment (except positive and negative)

Let’s plot the wordcloud for each category

Let us check top words that depicts “positive” and “negative” sentiments in NRC Sentiment category.

Joining, by = "word"
Joining, by = "word"

Let us check top words that depicts “positive” and “negative” sentiments in bing Sentiment category.

Joining, by = "word"
Joining, by = "word"
Joining, by = "word"
Joining, by = "word"

It can be seen that positive and negative words are different for all three lexicons.

Let’s plot positive and negative word cloud

Let’s check songs based on NRC sentiment (except positive and negative)

lyrics_words <- select(lyrics, c("artist", "song","text"))
lyrics_words <- lyrics_words %>% unnest_tokens(word, text)
head(lyrics_words)

dim(lyrics_words)

[1] 13186646        3

joy_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "joy")) 

Joining, by = "word"

joy_s <- as.data.frame(sort(table(joy_s$song)))
columns_sentiment <- c("Song", "Freq")
colnames(joy_s) <- columns_sentiment
tail(joy_s, 10)

trust_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "trust")) 

Joining, by = "word"

trust_s <- as.data.frame(sort(table(trust_s$song)))
colnames(trust_s) <- columns_sentiment
tail(trust_s, 10)

fear_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "fear")) 

Joining, by = "word"

fear_s <- as.data.frame(sort(table(fear_s$song)))
colnames(fear_s) <- columns_sentiment
sadness_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "sadness")) 

Joining, by = "word"

sadness_s <- as.data.frame(sort(table(sadness_s$song)))
colnames(sadness_s) <- columns_sentiment
anger_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "anger")) 

Joining, by = "word"

anger_s <- as.data.frame(sort(table(anger_s$song)))
colnames(anger_s) <- columns_sentiment
surprise_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "surprise")) 

Joining, by = "word"

surprise_s <- as.data.frame(sort(table(surprise_s$song)))
colnames(surprise_s) <- columns_sentiment
disgust_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "disgust")) 

Joining, by = "word"

disgust_s <- as.data.frame(sort(table(disgust_s$song)))
colnames(disgust_s) <- columns_sentiment
anticipation_s <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "anticipation")) 

Joining, by = "word"

anticipation_s <- as.data.frame(sort(table(anticipation_s$song)))
colnames(anticipation_s) <- columns_sentiment

Let’s plot the top 10 songs in each category

Let’s do the same for Positive and Negative sentiment in NRC lexicon cateogory.

Joining, by = "word"
Joining, by = "word"

Let us find positive and negative songs based on bing and loughran lexicon cateogories.

Joining, by = "word"
Joining, by = "word"
Joining, by = "word"
Joining, by = "word"

The positive and negative songs in each category are different. This is something known already, since the positive words and negative words in ech categories were also different.

The same analysis can be done for artists. Let’s check out artists who write most songs in each NRC lexicon sentiment cateogry.

joy_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "joy")) 

Joining, by = "word"

joy_a <- as.data.frame(sort(table(joy_a$artist)))
columns_sentiment <- c("artist", "Freq")
colnames(joy_a) <- columns_sentiment
tail(joy_a, 10)

trust_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "trust")) 

Joining, by = "word"

trust_a <- as.data.frame(sort(table(trust_a$artist)))
colnames(trust_a) <- columns_sentiment
tail(trust_a, 10)

fear_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "fear")) 

Joining, by = "word"

fear_a <- as.data.frame(sort(table(fear_a$artist)))
colnames(fear_a) <- columns_sentiment
sadness_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "sadness")) 

Joining, by = "word"

sadness_a <- as.data.frame(sort(table(sadness_a$artist)))
colnames(sadness_a) <- columns_sentiment
anger_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "anger")) 

Joining, by = "word"

anger_a <- as.data.frame(sort(table(anger_a$artist)))
colnames(anger_a) <- columns_sentiment
surprise_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "surprise")) 

Joining, by = "word"

surprise_a <- as.data.frame(sort(table(surprise_a$artist)))
colnames(surprise_a) <- columns_sentiment
disgust_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "disgust")) 

Joining, by = "word"

disgust_a <- as.data.frame(sort(table(disgust_a$artist)))
colnames(disgust_a) <- columns_sentiment
anticipation_a <- lyrics_words %>%
  inner_join(get_sentiments("nrc") %>% 
  filter(sentiment == "anticipation")) 

Joining, by = "word"

anticipation_a <- as.data.frame(sort(table(anticipation_a$artist)))
colnames(anticipation_a) <- columns_sentiment

Let’s plot the top 10 artists in each category

Let’s do the same for Positive and Negative sentiment in NRC lexicon cateogory.

Joining, by = "word"
Joining, by = "word"

Let us find positive and negative songs based on bing and loughran lexicon cateogories.

Joining, by = "word"
Joining, by = "word"
Joining, by = "word"
Joining, by = "word"

Let’s check out bigrams in the songs lyrics

Error in unnest_tokens(lyrics, input = text, output = bigram, token = "ngrams",  : 
  could not find function "unnest_tokens"

  
bigram_counts <- bigram_united %>% 
  count(bigram, sort = TRUE)
  head(bigram_counts) 
  
   
  bigram_counts %>% arrange(desc(n))%>% head(20)%>%ggplot(aes(x=factor(bigram,levels=bigram),y=n))+geom_bar(stat="identity",fill="#FF3E45")+labs(title="Top 20 bigram words in Songs")+coord_flip()
  
 
  wordcloud(bigram_counts$bigram,bigram_counts$n,max.words=50,random.order = F,colors=c("#9E0142", "#D53E4F", "#F46D43" ,"#FDAE61", "#FEE08B", "#E6F598" ,"#ABDDA4" ,"#66C2A5" ,"#3288BD", "#5E4FA2"))
title(main="Bigram wordcloud")